/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package hiddenwebsearch;

import java.util.*;
import java.io.*;

/**
 *
 * @author Star
 */
public final class QProbe {
    //define Database information
    private int DBSize = 0;
    public String DBUrl = "";
    //current category
    private String category = "";
    private String appid = "5345086B524C8E1EC50ABFA9475AA09E5BE2A4F5";
    //define query word map,url map for each query, Node map for each category
    private HashMap<String, List<String>> qMap = new HashMap<String, List<String>>();
    private HashMap<String, List<String>> uMap = new HashMap<String, List<String>>();
    private HashMap<String, CNode> nMap = new HashMap<String, CNode>();
    //define category list
    private List<String> clist = new ArrayList<String>();
    //initialize category tree
    private CNode root = CNode.getDefault();
    //initialize result node
    private CNode categoryListRoot = new CNode("");
    //ts and tc for the db
    private double ts = 0;
    private int tc = 0;


    public QProbe() {
    }
    // Initialize the QProbe, set initial host, Query word list and Database size
    
    public QProbe(double ts, int tc, String url) {
        this.setQueryList();
        this.ts = ts;
        this.tc = tc;
        this.DBUrl = url;
        this.getDBSize();
        if (DBSize < tc) {
            System.out.println("0 category! Change Parameters!");
        } else {
            categoryListRoot.value = "Root";
            categoryListRoot.Keyword = this.getQuery("Root");
            clist.add("Root");
        }
    }
    
    public void setQueryList() {
        setQueryListByName("./Text/root.txt");
    }

    //Read query words from root.txt set the query words for each category.
    public void setQueryListByName(String filename) {
        File file = new File(filename);
        BufferedReader reader = null;
        category = "";
        try {
            reader = new BufferedReader(new FileReader(file));
            String tempString = null;
            List<String> tmplist = new ArrayList<String>();
            while ((tempString = reader.readLine()) != null && tempString.length() > 0) {
                    String[] ts = tempString.split(" ");
                    String keyword = "";
                    //if it is the first line or the same category as before, add to old list
                    if (category == "" || ts[0].equals(category)) {
                        category = ts[0];
                        if (ts.length > 2) {
                            for (int i = 1; i < ts.length; i++) {
                                keyword += ts[i] + " ";
                            }
                            keyword = keyword.substring(0, keyword.length() - 1);

                        } else {
                            keyword = ts[1];
                        }
                        tmplist.add(keyword);
                    } 
                    //if it is a new category, put the old list into hashmap
                    else if (!(ts[0].equals(category))) {
                        List<String> catelist = new ArrayList<String>();
                        catelist.addAll(tmplist);
                        qMap.put(category, catelist);
                        category = ts[0];
                        if (ts.length > 2) {
                            keyword = ts[1] + "+" + ts[2];
                        } else {
                            keyword = ts[1];
                        }
                        tmplist.clear();
                        tmplist.add(keyword);
                    }
                }
            //put the last one into Hashmap
            qMap.put(category, tmplist);
            //set the query word list for root
            List<String> rootlist = new ArrayList<String>();
            rootlist.addAll(qMap.get("Computers"));
            rootlist.addAll(qMap.get("Health"));
            rootlist.addAll(qMap.get("Sports"));
            qMap.put("Root", rootlist);
            
            reader.close();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if (reader != null) {
                try {
                    reader.close();
                } catch (IOException e1) {
                }
            }
        }
    }
    
   //get Query list from category name 
    public List<String> getQuery(String cag) {
        return this.qMap.get(cag);
    }
    
    //generate the query url for each query word
    public String setQuery(String keyword, String db) {
        String url = "http://api.bing.net/xml.aspx?AppId=" + appid + "&Sources=Web&Market=en-us&Adult=Moderate&Web.Options=DisableHostCollapsing+DisableQueryAlterations&Query=";
        url += "site:" + db + " "+keyword;
        return url;
    }

    //get coverage for a certain category
    private int getCategoryInfo(String category, String dburl) {
        int size = 0;
        //if data already exist, read from hashmap
        if (nMap.get(category) != null) {
            CNode node = nMap.get(category);
            size = node.tc;
        } else {
            //Use urlparser to get the size and url list for a certain query
            URLParser up = new URLParser();
            String query = "";
            List<String> qlist = qMap.get(category);
            for (int i = 0; i < qlist.size(); i++) {
                List<String> urlList = new ArrayList<String>();
                query = this.setQuery(qlist.get(i), dburl);
                up.parserXml(query);
                urlList.addAll(up.getUrls());
                size += up.getSize();
                //store the url list for a certain query
                uMap.put(qlist.get(i), urlList);
                CNode node = new CNode(category);
                node.tc=size;
                //store the information for a certain category node
                nMap.put(category, node);
            }
        }
        return size;
    }

    private int getDBSize() {
        int size = 0;
        //initial the DB size, the sum of three subcategory.
        size=getCategoryInfo("Computers",DBUrl)+getCategoryInfo("Health",DBUrl)+getCategoryInfo("Sports",DBUrl);
        DBSize = size;
        return DBSize;
    }
    
    //return a URL list of a certain keywords 
    public List<String> getURL(String key) {
        if (uMap.get(key) != null) {
            return uMap.get(key);
        } else {
            //return a url with query
            String url = this.setQuery(key, DBUrl);
            URLParser up = new URLParser();
            up.parserXml(url);
            List<String> url_list = up.getUrls();
            return url_list;
        }
    }
    //Calculate the specificity, calculte the protion and multiple the specificity from parent
    private double getSpec(String category) {
        double c = this.getCategoryInfo(category, DBUrl);
        CNode node = nMap.get(category);
        
        if (category.equals("Hardware") || category.equals("Programming")) {
            c = c / (this.getCategoryInfo("Hardware", DBUrl) + this.getCategoryInfo("Programming", DBUrl))*nMap.get("Computers").ts;
        }
        if (category.equals("Fitness") || category.equals("Diseases")) {
            c = c / (this.getCategoryInfo("Fitness", DBUrl) + this.getCategoryInfo("Diseases", DBUrl))*nMap.get("Health").ts;
        }

        if (category.equals("Basketball") || category.equals("Soccer")) {
            c = c / (this.getCategoryInfo("Basketball", DBUrl) + this.getCategoryInfo("Soccer", DBUrl))*nMap.get("Sports").ts;
        }
         if (category.equals("Computers") || category.equals("Health") || category.equals("Sports")) {
            c = c / DBSize;
            
        }
        //update the ts for this category node 
        node.ts=c;
        return c;
    }

    //classify a databse into different category
    public void classify(CNode node, CNode root) {
        String cag = node.value;
        //get coverage and Specificity
        int ctc = this.getCategoryInfo(cag, DBUrl);
        double cts = this.getSpec(cag);
        
        System.out.println("Category:" + cag + " Specificity:" + cts + " Coverage:" + ctc);
        //if satisify the ts and tc, then detect next level 
        if (cts >= ts && ctc > tc) {
            //Save the keywords for summuary
            CNode tmp = new CNode(cag);
            if (node.value.equals("Computers")) {
                tmp.Keyword.addAll(this.getQuery("Programming"));
                tmp.Keyword.addAll(this.getQuery("Hardware"));
            }
            if (node.value.equals("Health")) {
                tmp.Keyword.addAll(this.getQuery("Fitness"));
                tmp.Keyword.addAll(this.getQuery("Diseases"));
            }
            if (node.value.equals("Sports")) {
                tmp.Keyword.addAll(this.getQuery("Soccer"));
                tmp.Keyword.addAll(this.getQuery("Basketball"));
            }
            
            root.add(tmp);
            root.Keyword = root.Merge(tmp.Keyword);
            clist.add(cag);
            //if the node has children, detect next level
            if (node.Children.size() > 0) {
                for (int i = 0; i < node.Children.size(); i++) {
                    classify(node.Children.get(i), tmp);
                }
            }
        }
    }
    
    //get the result node, used for summuary
    public CNode getRootnode()
    {
        for (int i = 0; i < this.root.Children.size(); i++) {
            this.classify(this.root.Children.get(i), this.categoryListRoot);
        }
        for (int i = 0; i < this.clist.size(); i++) {
            System.out.print(" "+clist.get(i));
        }
        return this.categoryListRoot;
        
    }
 
 
}
